# load packages
library(dplyr)
library(ggplot2)
library(chron)
library(plotly)
library(purrr)
library(RColorBrewer)
library(readr)

cb_10M_1973_2005_weather <- read_csv("cb_10M_1973_2005_weather.csv", show_col_types = FALSE)

cb_10M_2006_2019_weather <- read_csv("cb_10M_2006_2019_weather.csv", show_col_types = FALSE)

clean_data <- function(df) {
  df <- dplyr::select(df, Year, Name, Age, 
               Time, Division, pos_by_sex, 
               total_by_sex, Sex, PRCP, TMAX, TMIN)

  df$Age[df$Age == 'NR'] <- 0
  df$Time[df$Time == 'NR'] <- 0

df <- df %>%
    mutate(
      Year = as.integer(Year),
      Name = as.character(Name),
      Age = as.integer(Age, na.rm = TRUE),
      Time = as.character(Time),
      Division = as.character(Division),
      pos_by_sex = as.integer(pos_by_sex),
      total_by_sex = as.integer(total_by_sex),
      Sex = as.character(Sex),
      PRCP = as.numeric(PRCP),
      TMAX = as.integer(TMAX),
      TMIN = as.integer(TMIN)
    )
  
  return(df)
}

df1 <- clean_data(cb_10M_1973_2005_weather)
df2 <- clean_data(cb_10M_2006_2019_weather)


df <- bind_rows(df1, df2)

# fixes times with mins recorded as hrs (mm:ss:00 -> 00:mm:ss)
df <- df %>%
  mutate(
    Time = ifelse(
      Time >= "43:00:00" & Time <= "59:59:59",
      paste0("00:", substr(Time, 1, 2), ":", substr(Time, 4, 5)),
      Time
    )
  )


df <- df %>%
  filter(!is.na(Time) & Time != "" & Time != "00:00:00") %>%
  mutate(Time = chron::times(Time)) %>%
  filter(!is.na(Time)) %>%
  filter(Time >= "00:43:00" & Time <= "02:20:00") %>%
  filter(Age >= 8 & Age <= 87) %>%
  filter(Year != 1977) %>%
  filter(Year != 1973) %>%
  mutate(
    # replaces 'W' with 'F' in sex col
    Sex = ifelse(Sex == 'W', 'F', Sex),
    # correction for missing values by first character in division
    Sex = ifelse(substr(Division, 1, 1) == 'W', 'F', Sex),
    Sex = ifelse(substr(Division, 1, 1) == 'M', 'M', Sex)
  )

# add final data to git repository
# write.csv(df, "cleaned_10M_1973_2019.csv", row.names = FALSE)

unique(df$Division)